library(NNbenchmark)
library(kableExtra)
library(knitr)
options(scipen = 999)
NNdataSummary(NNbigdatasets)
## Dataset n_rows n_inputs n_neurons n_parameters
## 1 bWoodN1 20000 6 5 41
ht(bWoodN1)
## x1 x2 x3 x4 x5 x6 y
## 1 0.5042050 0.05765295 0.9028320 0.4322548 0.4187889 0.3419781 -15.178295
## 2 0.9301722 0.53164816 0.4434929 0.8087859 0.6993847 0.5547075 14.048025
## 3 0.3437977 0.22566605 0.6483183 0.5155759 0.9063721 0.4779758 -20.257463
## 19998 1.0000000 1.00000000 1.0000000 1.0000000 0.0000000 1.0000000 24.227091
## 19999 1.0000000 1.00000000 1.0000000 1.0000000 1.0000000 0.0000000 5.560786
## 20000 1.0000000 1.00000000 1.0000000 1.0000000 1.0000000 1.0000000 24.309251
if(dir.exists("D:/GSoC2020/Results/2020run05/"))
{
odir <- "D:/GSoC2020/Results/2020run05/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{
odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
odir <- "."
nrep <- 10
maxit2ndorder <- 400
maxit1storderA <- 2000
maxit1storderB <- 10000
maxit1storderC <- 100000
Extract appropriate R code for NN configuration
pkglist <- c("nlsr", "rminer", "nnet", "validann", "MachineShop")
for(pkg in pkglist)
{
pkgname.R <- paste0("NNbenchmark-2020-", pkg, ".R")
pkgname.Rmd <- paste0("NNbenchmark-2020-", pkg, ".Rmd")
configname.R <- paste0("NNbenchmark-2020-", pkg, "-Config.R")
if(!file.exists(configname.R))
{
if(!file.exists(pkgname.R))
purl(pkgname.Rmd)
ConfigFile <- readLines(pkgname.R)
ConfigFile.beg <- 28
ConfigFile.end <- which(ConfigFile == "if(FALSE)")-1
if(length(ConfigFile.end) == 0)
ConfigFile.end <- 48
writeLines(ConfigFile[ConfigFile.beg:ConfigFile.end], configname.R)
}
source(configname.R)
}
methodlist <- list(nlsr.method,
rminer.method,
nnet.method,
validann.method[2],
MachineShop.method)
pkgfunmat <- rbind(c("nlsr", "nlxb"),
c("rminer", "fit"),
c("nnet", "nnet"),
c("validann","ann"),
c("MachineShop", "fit"))
colnames(pkgfunmat) <- c("pkg", "fun")
trainvect <- paste("NNtrain", pkgfunmat[,"pkg"], sep=".")
hypervect <- paste("hyperParams", pkgfunmat[,"pkg"], sep=".")
predvect <- paste("NNpredict", pkgfunmat[,"pkg"], sep=".")
closevect <- paste("NNclose", pkgfunmat[,"pkg"], sep=".")
startvect <- rep(NA, length(pkgfunmat[,"pkg"]))
startvect[pkgfunmat[,"pkg"] == "h2o"] <- "NNstart.h2o"
preparelist <- list(nlsr.prepareZZ,
rminer.prepareZZ,
nnet.prepareZZ,
validann.prepareZZ,
MachineShop.prepareZZ)
names(preparelist) <- pkgfunmat[,"pkg"]
resall <- trainPredict_1data(dset=13, method=methodlist, train=trainvect, hyper=hypervect,
pred=predvect, summary=NNsummary, close=closevect,
start=startvect, prepare=preparelist, nrep=nrep, echo=FALSE, doplot=TRUE,
pkgname=pkgfunmat[,"pkg"], pkgfun=pkgfunmat[,"fun"],
csvfile = TRUE, rdafile = TRUE, odir = odir)
## vn: [1] "y" "b1" "b2" "b3" "b4" "x1" "b5" "x2" "b6" "x3" "b7" "x4"
## [13] "b8" "x5" "b9" "x6" "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y" "b1" "b2" "b3" "b4" "x1" "b5" "x2" "b6" "x3" "b7" "x4"
## [13] "b8" "x5" "b9" "x6" "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y" "b1" "b2" "b3" "b4" "x1" "b5" "x2" "b6" "x3" "b7" "x4"
## [13] "b8" "x5" "b9" "x6" "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y" "b1" "b2" "b3" "b4" "x1" "b5" "x2" "b6" "x3" "b7" "x4"
## [13] "b8" "x5" "b9" "x6" "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y" "b1" "b2" "b3" "b4" "x1" "b5" "x2" "b6" "x3" "b7" "x4"
## [13] "b8" "x5" "b9" "x6" "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y" "b1" "b2" "b3" "b4" "x1" "b5" "x2" "b6" "x3" "b7" "x4"
## [13] "b8" "x5" "b9" "x6" "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y" "b1" "b2" "b3" "b4" "x1" "b5" "x2" "b6" "x3" "b7" "x4"
## [13] "b8" "x5" "b9" "x6" "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y" "b1" "b2" "b3" "b4" "x1" "b5" "x2" "b6" "x3" "b7" "x4"
## [13] "b8" "x5" "b9" "x6" "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y" "b1" "b2" "b3" "b4" "x1" "b5" "x2" "b6" "x3" "b7" "x4"
## [13] "b8" "x5" "b9" "x6" "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y" "b1" "b2" "b3" "b4" "x1" "b5" "x2" "b6" "x3" "b7" "x4"
## [13] "b8" "x5" "b9" "x6" "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## initial value 17273.889382
## iter 20 value 7814.839497
## iter 40 value 6956.452168
## iter 60 value 3562.078201
## iter 80 value 2876.956130
## iter 100 value 2771.727490
## iter 120 value 2725.413319
## iter 140 value 2309.946563
## iter 160 value 2227.702766
## iter 180 value 2153.534988
## iter 200 value 2136.926477
## iter 220 value 2112.408209
## iter 240 value 2105.505830
## iter 260 value 2104.297916
## iter 280 value 2103.699287
## iter 300 value 2102.684227
## iter 320 value 2102.024712
## final value 2101.647096
## converged
## initial value 19189.354929
## iter 20 value 7168.639955
## iter 40 value 6306.559542
## iter 60 value 5126.481393
## iter 80 value 4885.143722
## iter 100 value 4606.138523
## iter 120 value 4421.788302
## iter 140 value 2835.098370
## iter 160 value 2532.582809
## iter 180 value 2434.206455
## iter 200 value 2356.566537
## iter 220 value 2202.087874
## iter 240 value 2173.312389
## iter 260 value 2152.990275
## iter 280 value 2150.333116
## iter 300 value 2142.629340
## iter 320 value 2132.767767
## iter 340 value 2128.817690
## iter 360 value 2128.562815
## iter 380 value 2124.399055
## iter 400 value 2121.846377
## final value 2121.846377
## stopped after 400 iterations
## initial value 22682.675810
## iter 20 value 7107.439389
## iter 40 value 6407.452943
## iter 60 value 5016.488014
## iter 80 value 4624.433805
## iter 100 value 4406.760205
## iter 120 value 4304.703921
## iter 140 value 3821.299198
## iter 160 value 3729.268331
## iter 180 value 3698.706733
## iter 200 value 3695.749947
## iter 220 value 3685.929206
## iter 240 value 3681.289933
## iter 260 value 3680.287723
## iter 280 value 3679.738781
## iter 300 value 3679.317716
## iter 320 value 3679.020653
## final value 3678.954283
## converged
## initial value 17431.623442
## iter 20 value 7982.028365
## iter 40 value 6654.263756
## iter 60 value 5120.523807
## iter 80 value 4273.500686
## iter 100 value 4182.557045
## iter 120 value 4138.598850
## iter 140 value 4080.165545
## iter 160 value 3946.228255
## iter 180 value 3618.169004
## iter 200 value 3374.211115
## iter 220 value 3334.049353
## iter 240 value 3331.314196
## iter 260 value 3312.526580
## iter 280 value 3305.626456
## iter 300 value 3303.900993
## iter 320 value 3303.518441
## iter 340 value 3302.689444
## iter 360 value 3301.605187
## iter 380 value 3301.444835
## iter 380 value 3301.444832
## final value 3301.444547
## converged
## initial value 31586.837042
## iter 20 value 7998.405040
## iter 40 value 6370.263463
## iter 60 value 4571.258707
## iter 80 value 4194.798333
## iter 100 value 3979.754828
## iter 120 value 3921.905987
## iter 140 value 3793.076128
## iter 160 value 3772.226238
## iter 180 value 3765.867233
## iter 200 value 3763.466999
## iter 220 value 3757.746328
## iter 240 value 3753.872161
## iter 260 value 3752.850981
## iter 280 value 3752.790531
## iter 300 value 3752.178415
## iter 320 value 3751.648513
## iter 340 value 3751.405254
## final value 3751.402951
## converged
## initial value 23231.667172
## iter 20 value 8248.993250
## iter 40 value 6573.172905
## iter 60 value 5255.382651
## iter 80 value 4495.747877
## iter 100 value 4343.388469
## iter 120 value 4165.253399
## iter 140 value 3464.479214
## iter 160 value 2274.256697
## iter 180 value 2136.165376
## iter 200 value 2066.458897
## iter 220 value 1907.752045
## iter 240 value 1892.091609
## iter 260 value 1885.460371
## iter 280 value 1885.186693
## iter 300 value 1884.450528
## iter 320 value 1881.350289
## iter 340 value 1880.507537
## final value 1880.482945
## converged
## initial value 19436.122157
## iter 20 value 9188.718559
## iter 40 value 8336.494118
## iter 60 value 5390.863748
## iter 80 value 5026.150880
## iter 100 value 4992.630757
## iter 120 value 4981.259370
## iter 140 value 4937.314714
## iter 160 value 4871.393409
## iter 180 value 4676.488264
## iter 200 value 4659.870554
## iter 220 value 4584.433558
## iter 240 value 4574.271196
## final value 4574.017564
## converged
## initial value 27725.438510
## iter 20 value 8528.333610
## iter 40 value 7689.246161
## iter 60 value 3970.671324
## iter 80 value 2994.272014
## iter 100 value 2763.081398
## iter 120 value 2705.952817
## iter 140 value 2637.525815
## iter 160 value 2601.802989
## iter 180 value 2497.644434
## iter 200 value 2460.944997
## iter 220 value 2159.258009
## iter 240 value 2144.580184
## iter 260 value 2130.865033
## iter 280 value 2128.186012
## iter 300 value 2120.435747
## iter 320 value 2117.243931
## iter 340 value 2116.888886
## iter 360 value 2115.888045
## iter 380 value 2115.246899
## iter 400 value 2114.980559
## final value 2114.980559
## stopped after 400 iterations
## initial value 24018.488067
## iter 20 value 7941.706394
## iter 40 value 6817.451435
## iter 60 value 3914.857157
## iter 80 value 3211.179530
## iter 100 value 2721.907140
## iter 120 value 2674.391432
## iter 140 value 2288.033016
## iter 160 value 2185.058534
## iter 180 value 2166.867293
## iter 200 value 2164.441489
## iter 220 value 2147.752990
## iter 240 value 2137.179958
## iter 260 value 2133.977720
## iter 280 value 2133.839593
## iter 280 value 2133.839581
## final value 2133.839387
## converged
## initial value 22619.138407
## iter 20 value 7846.642262
## iter 40 value 7041.317923
## iter 60 value 4875.174263
## iter 80 value 4515.778943
## iter 100 value 4360.826562
## iter 120 value 4349.378277
## iter 140 value 4264.855278
## iter 160 value 4248.154110
## final value 4246.959304
## converged
colnames(resall) <- pkgfunmat[,"pkg"]
rownames(resall) <-
paste0(rep(c("RMSE", "MSE", "MAE", "WAE", "time"), length=5*nrep), ".", rep(1:nrep, each=5))
resmin <- sapply(1:NCOL(resall),
function(j)
{
idx.min <- which.min(resall[grep("RMSE.", rownames(resall)),j])
idx <- (idx.min-1)*5+1:5
resall[idx, j]
})
colnames(resmin) <- pkgfunmat[,"pkg"]
rownames(resmin) <- c("RMSE", "MSE", "MAE", "WAE", "time")
kable(resmin)
| nlsr | rminer | nnet | validann | MachineShop | |
|---|---|---|---|---|---|
| RMSE | 3.5482 | 3.5519 | 3.5512 | 3.3599 | 3.5467 |
| MSE | 12.5896 | 12.6158 | 12.6109 | 11.2892 | 12.5794 |
| MAE | 2.8515 | 2.8490 | 2.8562 | 2.6929 | 2.8554 |
| WAE | 13.9362 | 13.6683 | 13.8621 | 16.1344 | 14.1641 |
| time | 85.0930 | 11.4620 | 3.6490 | 139.0810 | 2.7100 |
which.median <- function(x)
{
n <- length(x)
if(n %% 2 == 1)
which(rank(x) == n/2+1)
else
which(rank(x) == n/2)
}
# resmedian <- sapply(1:NCOL(resall),
# function(j)
# {
# idx.min <- which.median(resall[grep("RMSE.", rownames(resall)),j])
# idx <- (idx.min-1)*5+1:5
# resall[idx, j]
# })
# colnames(resmedian) <- pkgfunmat[,"pkg"]
# rownames(resmedian) <- c("RMSE", "MSE", "MAE", "WAE", "time")
# kable(resmedian)